this_notebook_name = "BreastSegmentationStudy-TF2"

# Update this folder name for your computer

local_data_folder = "/Usersß/Josh Ehrlich/Courses/CISC881/Project/data"
overwrite_existing_data_files = False

# All results and output will be archived with this timestamp

import datetime
save_timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
print("Save timestamp: {}".format(save_timestamp))

# Learning parameters

import numpy as np

ultrasound_size = 128
num_classes = 2
num_epochs = 200
batch_size = 64
max_learning_rate = 0.02
min_learning_rate = 0.000001
regularization_rate = 0.0001
filter_multiplier = 15
class_weights = np.array([0.15, 0.85])
learning_rate_decay = (max_learning_rate - min_learning_rate) / num_epochs

# Training data augmentation parameters

max_shift_factor = 0.12
max_rotation_angle = 10
max_zoom_factor = 1.1
min_zoom_factor = 0.8

# Evaluation parameters

acceptable_margin_mm = 1.0
mm_per_pixel = 1.0

roc_thresholds = [0.9, 0.8, 0.7, 0.65, 0.6, 0.55, 0.5, 0.45, 0.4, 0.35, 0.3, 0.25, 0.2, 0.15, 0.1,
                  0.08, 0.06, 0.04, 0.02, 0.01,
                  0.008, 0.006, 0.004, 0.002, 0.001]

'''
Provide NxM numpy array to schedule cross validation
N rounds of validation will be performed, leaving out M patients in each for validation data
All values should be valid patient IDs, or negative. Negative values are ignored.

Example 1: a leave-one-out cross validation with 3 patients would look like this:
validation_schedule_patient = np.array([[0],[1],[2]])

Example 2: a leave-two-out cross validation on 10 patients would look like this:
validation_schedule_patient = np.array([[0,1],[2,3],[4,5],[6,7],[8,9]])

Example 3: leave-one-out cross validation with 3 patients, then training on all available data (no validation):
validation_schedule_patient = np.array([[0],[1],[2],[-1]])
'''
validation_schedule_patient = np.array([[0,1,2,3], [7,11,13,18], [9,14,21,31], [19,22,29,32], [20,23,27,30]])

# Uncomment for faster debugging

roc_thresholds = [0.8, 0.6, 0.4, 0.2, 0.1, 0.01, 0.001]
num_epochs = 200

Save timestamp: 2020-11-27_22-59-01

import os
from random import sample
from pathlib import Path

from ipywidgets import IntProgress
from IPython.display import display, HTML

import diskcache
import girder_client
import matplotlib.pyplot as plt
import pandas as pd

import tensorflow as tf

import ultrasound_batch_generator as generator
import evaluation_metrics

from girder_apikey import girder_apikey

# Import aigt modules

import sys
parent_folder = os.path.dirname(os.path.abspath(os.curdir))
sys.path.append(parent_folder)

import Models.segmentation_unet as unet
import utils

# Creating standard folders to save data and logs

data_arrays_fullpath, notebooks_save_fullpath, results_save_fullpath, models_save_fullpath, val_data_fullpath =\
    utils.create_standard_project_folders(local_data_folder)

# Fetching Girder data

girder_url = "https://pocus.cs.queensu.ca/api/v1"
data_csv_file = "BreastGirder.csv"
girder_key = girder_apikey

ultrasound_arrays_by_patients, segmentation_arrays_by_patients =\
    utils.load_girder_data(data_csv_file, data_arrays_fullpath, girder_url, girder_key)
    
n_patients = len(ultrasound_arrays_by_patients)

for i in range(n_patients):
    print("Patient {} has {} ultrasounds and {} segmentations".format(
        i, ultrasound_arrays_by_patients[i].shape[0], segmentation_arrays_by_patients[i].shape[0]))

Patient 0 has 379 ultrasounds and 379 segmentations
Patient 1 has 292 ultrasounds and 292 segmentations
Patient 2 has 426 ultrasounds and 426 segmentations
Patient 3 has 418 ultrasounds and 418 segmentations
Patient 4 has 442 ultrasounds and 442 segmentations
Patient 5 has 224 ultrasounds and 224 segmentations
Patient 6 has 272 ultrasounds and 272 segmentations
Patient 7 has 288 ultrasounds and 288 segmentations
Patient 8 has 175 ultrasounds and 175 segmentations
Patient 9 has 113 ultrasounds and 113 segmentations
Patient 10 has 237 ultrasounds and 237 segmentations
Patient 11 has 180 ultrasounds and 180 segmentations
Patient 12 has 5 ultrasounds and 5 segmentations
Patient 13 has 402 ultrasounds and 402 segmentations
Patient 14 has 295 ultrasounds and 295 segmentations
Patient 15 has 331 ultrasounds and 331 segmentations
Patient 16 has 145 ultrasounds and 145 segmentations
Patient 17 has 253 ultrasounds and 253 segmentations
Patient 18 has 158 ultrasounds and 158 segmentations
Patient 19 has 187 ultrasounds and 187 segmentations
Patient 20 has 135 ultrasounds and 135 segmentations
Patient 21 has 107 ultrasounds and 107 segmentations
Patient 22 has 238 ultrasounds and 238 segmentations
Patient 23 has 226 ultrasounds and 226 segmentations
Patient 24 has 162 ultrasounds and 162 segmentations
Patient 25 has 124 ultrasounds and 124 segmentations
Patient 26 has 209 ultrasounds and 209 segmentations
Patient 27 has 123 ultrasounds and 123 segmentations
Patient 28 has 181 ultrasounds and 181 segmentations
Patient 29 has 134 ultrasounds and 134 segmentations
Patient 30 has 145 ultrasounds and 145 segmentations
Patient 31 has 133 ultrasounds and 133 segmentations
Patient 32 has 179 ultrasounds and 179 segmentations

# Prepare validation rounds

if np.max(np.max(validation_schedule_patient)) > (n_patients - 1):
    raise Exception("Patient ID cannot be greater than {}".format(n_patients - 1))

num_validation_rounds = len(validation_schedule_patient)
print("Planning {} rounds of validation".format(num_validation_rounds))
for i in range(num_validation_rounds):
    print("Validation on patients {} in round {}".format(validation_schedule_patient[i], i))

Planning 5 rounds of validation
Validation on patients [0 1 2 3] in round 0
Validation on patients [ 7 11 13 18] in round 1
Validation on patients [ 9 14 21 31] in round 2
Validation on patients [19 22 29 32] in round 3
Validation on patients [20 23 27 30] in round 4

# Print training parameters, to archive them together with the notebook output.

time_sequence_start = datetime.datetime.now()

print("Timestamp for saved files: {}".format(save_timestamp))
print("\nTraining parameters")
print("Number of epochs:    {}".format(num_epochs))
print("Step size maximum:   {}".format(max_learning_rate))
print("Step size decay:     {}".format(learning_rate_decay))
print("Batch size:          {}".format(batch_size))
print("Regularization rate: {}".format(regularization_rate))
print("")
print("Saving validation predictions in: {}".format(val_data_fullpath))
print("Saving models in:                 {}".format(models_save_fullpath))

# ROC data will be saved in these containers

val_best_metrics    = dict()
val_fuzzy_metrics   = dict()
val_aurocs          = np.zeros(num_validation_rounds)
val_best_thresholds = np.zeros(num_validation_rounds)

# Perform validation rounds

for val_round_index in range(num_validation_rounds):
    
    # Prepare data arrays
    
    train_ultrasound_data = np.zeros(
        [0,
         ultrasound_arrays_by_patients[0].shape[1],
         ultrasound_arrays_by_patients[0].shape[2],
         ultrasound_arrays_by_patients[0].shape[3]])
    
    train_segmentation_data = np.zeros(
        [0,
         segmentation_arrays_by_patients[0].shape[1],
         segmentation_arrays_by_patients[0].shape[2],
         segmentation_arrays_by_patients[0].shape[3]])
    
    val_ultrasound_data = np.zeros(
        [0,
         ultrasound_arrays_by_patients[0].shape[1],
         ultrasound_arrays_by_patients[0].shape[2],
         ultrasound_arrays_by_patients[0].shape[3]])
    
    val_segmentation_data = np.zeros(
        [0,
         segmentation_arrays_by_patients[0].shape[1],
         segmentation_arrays_by_patients[0].shape[2],
         segmentation_arrays_by_patients[0].shape[3]])
    
    for patient_index in range(n_patients):
        if patient_index not in validation_schedule_patient[val_round_index]:
            train_ultrasound_data = np.concatenate((train_ultrasound_data,
                                                    ultrasound_arrays_by_patients[patient_index]))
            train_segmentation_data = np.concatenate((train_segmentation_data,
                                                      segmentation_arrays_by_patients[patient_index]))
        else:
            val_ultrasound_data = np.concatenate((val_ultrasound_data,
                                                 ultrasound_arrays_by_patients[patient_index]))
            val_segmentation_data = np.concatenate((val_segmentation_data,
                                                   segmentation_arrays_by_patients[patient_index]))
    
    n_train = train_ultrasound_data.shape[0]
    n_val = val_ultrasound_data.shape[0]
    
    print("\n*** Leave-one-out round # {}".format(val_round_index))
    print("    Training on {} images, validating on {} images...".format(n_train, n_val))
    
    val_segmentation_data_onehot = tf.keras.utils.to_categorical(val_segmentation_data, num_classes)
    
    # Create and train model
    
    model = unet.segmentation_unet(ultrasound_size, num_classes, filter_multiplier, regularization_rate)
    
    model.compile(
        optimizer=tf.keras.optimizers.Adam(lr=max_learning_rate, decay=learning_rate_decay),
        loss=unet.weighted_categorical_crossentropy(class_weights),
        metrics=["accuracy"]
    )
    """
    #----------------------------------------------------------------------------
    import os
    import sys
    import datetime
    from random import sample
    from pathlib import Path
    import girder_client
    import matplotlib.pyplot as plt
    import pandas as pd
    import tensorflow as tf
    import numpy as np

    from Spine.ultrasound_batch_generator import train_preprocess, train_preprocess_with_maps, generate_weight_maps
    from Spine import evaluation_metrics

    from Spine.models import (
        new_unet,
        weighted_categorical_crossentropy,
        weighted_categorical_crossentropy_with_maps,
    )
    import utils


    batch_size=128, 
    num_epochs=100, 
    sagittal_only=False, 
    num_frames=1, 
    with_maps=False, 
    learning_rate=0.002,
    lr_decay=False,
    dropout=0.0,
    use_attention=True,
    num_layers=5,
    filters=16,
    use_batch_norm=True,
    load_from_save=False,

    ultrasound_size = 128
    batch_size = 128
    num_classes = 2
    min_learning_rate = 0.00001
    class_weights = np.array([0.1, 0.9])
    learning_rate_decay = (0.002-0.00001) / 100
    num_epochs = 100

    model = new_unet(
                input_size = ultrasound_size,
                num_classes=num_classes,
                num_channels=num_frames,
                use_batch_norm=use_batch_norm,
                upsample_mode="deconv",  # 'deconv' or 'simple'
                dropout=dropout,
                dropout_type="spatial",
                use_attention=use_attention,
                filters=filters,
                num_layers=num_layers,
                output_activation="softmax",
            )

    learning_rate = 0.002
    loss_func = weighted_categorical_crossentropy(class_weights)
    
    preprocess_func = train_preprocess
    print(learning_rate, learning_rate_decay)
    model.compile(
                optimizer=tf.keras.optimizers.Adam(
                    lr=learning_rate, decay=learning_rate_decay
                ),
                loss=loss_func,
                metrics=["accuracy", evaluation_metrics.jaccard_coef, evaluation_metrics.dice_coef],
            )

#-----------------------------------------------------
    """
    
    model.summary() #prints the structure of the neural network. If you change the unet,
    #print this so yuo can see it.

    training_generator = generator.UltrasoundSegmentationBatchGenerator(
        train_ultrasound_data,
        train_segmentation_data[:, :, :, 0],
        batch_size,
        (ultrasound_size, ultrasound_size),
        max_shift_factor=max_shift_factor,
        min_zoom_factor=min_zoom_factor,
        max_zoom_factor=max_zoom_factor,
        max_rotation_angle=max_rotation_angle
    )
        
    training_time_start = datetime.datetime.now()
    print("TRAINING LOG",training_generator)
    if n_val > 0:
        training_log = model.fit_generator(
            training_generator,
            validation_data=(val_ultrasound_data, val_segmentation_data_onehot),
            epochs=num_epochs,
            verbose=0)
    else:
        training_log = model.fit_generator(training_generator, epochs=num_epochs, verbose=0)
    
    training_time_stop = datetime.datetime.now()
    
    # Pring training log
    
    print("  Training time: {}".format(training_time_stop-training_time_start))
    
    # Plot training loss and metrics
    
    fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(12, 4))
    
    axes[0].plot(training_log.history['loss'], 'bo--')
    if n_val > 0:
        axes[0].plot(training_log.history['val_loss'], 'ro-')
    axes[0].set(xlabel='Epochs (n)', ylabel='Loss')
    if n_val > 0:
        axes[0].legend(['Training loss', 'Validation loss'])
    
    axes[1].plot(training_log.history['accuracy'], 'bo--')
    if n_val > 0:
        axes[1].plot(training_log.history['val_accuracy'], 'ro-')
    axes[1].set(xlabel='Epochs (n)', ylabel='Accuracy')
    if n_val > 0:
        axes[1].legend(['Training accuracy', 'Validation accuracy'])
    
    fig.tight_layout()
    
    # Archive trained model with unique filename based on notebook name and timestamp
    model_file_name = this_notebook_name + "_model-" + str(val_round_index) + "_" + save_timestamp + ".h5"
    model_fullname = os.path.join(models_save_fullpath, model_file_name)
    model.save(model_fullname)

    # Predict on validation data
    
    if n_val > 0:
        print(val_ultrasound_data.shape)
        y_pred_val  = model.predict(val_ultrasound_data)

        # Saving predictions for further evaluation

        val_prediction_filename = save_timestamp + "_prediction_" + str(val_round_index) + ".npy"
        val_prediction_fullname = os.path.join(val_data_fullpath, val_prediction_filename)
        np.save(val_prediction_fullname, y_pred_val)
        
        # Validation results

        vali_metrics_dicts, vali_best_threshold_index, vali_area = evaluation_metrics.compute_roc(
            roc_thresholds, y_pred_val, val_segmentation_data, acceptable_margin_mm, mm_per_pixel)

        val_fuzzy_metrics[val_round_index] = evaluation_metrics.compute_evaluation_metrics(
            y_pred_val, val_segmentation_data, acceptable_margin_mm, mm_per_pixel)

        val_best_metrics[val_round_index]    = vali_metrics_dicts[vali_best_threshold_index]
        val_aurocs[val_round_index]          = vali_area
        val_best_thresholds[val_round_index] = roc_thresholds[vali_best_threshold_index]
    
    # Printing total time of this validation round
    
    print("\nTotal round time:  {}".format(datetime.datetime.now() - training_time_start))
    print("")


time_sequence_stop = datetime.datetime.now()

print("\nTotal training time:   {}".format(time_sequence_stop - time_sequence_start))

Timestamp for saved files: 2020-11-27_22-59-01

Training parameters
Number of epochs:    200
Step size maximum:   0.02
Step size decay:     9.9995e-05
Batch size:          64
Regularization rate: 0.0001

Saving validation predictions in: /Usersß/Josh Ehrlich/Courses/CISC881/Project/data\PredictionsValidation
Saving models in:                 /Usersß/Josh Ehrlich/Courses/CISC881/Project/data\SavedModels

*** Leave-one-out round # 0
    Training on 5803 images, validating on 1515 images...
Model: "functional_1"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_1 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d (Conv2D)                 (None, 64, 64, 8)    80          input_1[0][0]                    
__________________________________________________________________________________________________
max_pooling2d (MaxPooling2D)    (None, 64, 64, 8)    0           conv2d[0][0]                     
__________________________________________________________________________________________________
conv2d_1 (Conv2D)               (None, 32, 32, 16)   1168        max_pooling2d[0][0]              
__________________________________________________________________________________________________
max_pooling2d_1 (MaxPooling2D)  (None, 32, 32, 16)   0           conv2d_1[0][0]                   
__________________________________________________________________________________________________
conv2d_2 (Conv2D)               (None, 16, 16, 32)   4640        max_pooling2d_1[0][0]            
__________________________________________________________________________________________________
max_pooling2d_2 (MaxPooling2D)  (None, 16, 16, 32)   0           conv2d_2[0][0]                   
__________________________________________________________________________________________________
conv2d_3 (Conv2D)               (None, 8, 8, 64)     18496       max_pooling2d_2[0][0]            
__________________________________________________________________________________________________
max_pooling2d_3 (MaxPooling2D)  (None, 8, 8, 64)     0           conv2d_3[0][0]                   
__________________________________________________________________________________________________
conv2d_4 (Conv2D)               (None, 4, 4, 128)    73856       max_pooling2d_3[0][0]            
__________________________________________________________________________________________________
max_pooling2d_4 (MaxPooling2D)  (None, 4, 4, 128)    0           conv2d_4[0][0]                   
__________________________________________________________________________________________________
conv2d_5 (Conv2D)               (None, 2, 2, 256)    295168      max_pooling2d_4[0][0]            
__________________________________________________________________________________________________
max_pooling2d_5 (MaxPooling2D)  (None, 2, 2, 256)    0           conv2d_5[0][0]                   
__________________________________________________________________________________________________
conv2d_6 (Conv2D)               (None, 1, 1, 512)    1180160     max_pooling2d_5[0][0]            
__________________________________________________________________________________________________
max_pooling2d_6 (MaxPooling2D)  (None, 1, 1, 512)    0           conv2d_6[0][0]                   
__________________________________________________________________________________________________
up_sampling2d (UpSampling2D)    (None, 2, 2, 512)    0           max_pooling2d_6[0][0]            
__________________________________________________________________________________________________
concatenate (Concatenate)       (None, 2, 2, 768)    0           up_sampling2d[0][0]              
                                                                 max_pooling2d_5[0][0]            
__________________________________________________________________________________________________
conv2d_7 (Conv2D)               (None, 2, 2, 506)    6218234     concatenate[0][0]                
__________________________________________________________________________________________________
batch_normalization (BatchNorma (None, 2, 2, 506)    2024        conv2d_7[0][0]                   
__________________________________________________________________________________________________
up_sampling2d_1 (UpSampling2D)  (None, 4, 4, 506)    0           batch_normalization[0][0]        
__________________________________________________________________________________________________
concatenate_1 (Concatenate)     (None, 4, 4, 634)    0           up_sampling2d_1[0][0]            
                                                                 max_pooling2d_4[0][0]            
__________________________________________________________________________________________________
conv2d_8 (Conv2D)               (None, 4, 4, 250)    2536250     concatenate_1[0][0]              
__________________________________________________________________________________________________
batch_normalization_1 (BatchNor (None, 4, 4, 250)    1000        conv2d_8[0][0]                   
__________________________________________________________________________________________________
up_sampling2d_2 (UpSampling2D)  (None, 8, 8, 250)    0           batch_normalization_1[0][0]      
__________________________________________________________________________________________________
concatenate_2 (Concatenate)     (None, 8, 8, 314)    0           up_sampling2d_2[0][0]            
                                                                 max_pooling2d_3[0][0]            
__________________________________________________________________________________________________
conv2d_9 (Conv2D)               (None, 8, 8, 122)    613050      concatenate_2[0][0]              
__________________________________________________________________________________________________
batch_normalization_2 (BatchNor (None, 8, 8, 122)    488         conv2d_9[0][0]                   
__________________________________________________________________________________________________
up_sampling2d_3 (UpSampling2D)  (None, 16, 16, 122)  0           batch_normalization_2[0][0]      
__________________________________________________________________________________________________
concatenate_3 (Concatenate)     (None, 16, 16, 154)  0           up_sampling2d_3[0][0]            
                                                                 max_pooling2d_2[0][0]            
__________________________________________________________________________________________________
conv2d_10 (Conv2D)              (None, 16, 16, 58)   142970      concatenate_3[0][0]              
__________________________________________________________________________________________________
batch_normalization_3 (BatchNor (None, 16, 16, 58)   232         conv2d_10[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_4 (UpSampling2D)  (None, 32, 32, 58)   0           batch_normalization_3[0][0]      
__________________________________________________________________________________________________
concatenate_4 (Concatenate)     (None, 32, 32, 74)   0           up_sampling2d_4[0][0]            
                                                                 max_pooling2d_1[0][0]            
__________________________________________________________________________________________________
conv2d_11 (Conv2D)              (None, 32, 32, 26)   30810       concatenate_4[0][0]              
__________________________________________________________________________________________________
batch_normalization_4 (BatchNor (None, 32, 32, 26)   104         conv2d_11[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_5 (UpSampling2D)  (None, 64, 64, 26)   0           batch_normalization_4[0][0]      
__________________________________________________________________________________________________
concatenate_5 (Concatenate)     (None, 64, 64, 34)   0           up_sampling2d_5[0][0]            
                                                                 max_pooling2d[0][0]              
__________________________________________________________________________________________________
conv2d_12 (Conv2D)              (None, 64, 64, 10)   5450        concatenate_5[0][0]              
__________________________________________________________________________________________________
batch_normalization_5 (BatchNor (None, 64, 64, 10)   40          conv2d_12[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_6 (UpSampling2D)  (None, 128, 128, 10) 0           batch_normalization_5[0][0]      
__________________________________________________________________________________________________
concatenate_6 (Concatenate)     (None, 128, 128, 11) 0           up_sampling2d_6[0][0]            
                                                                 input_1[0][0]                    
__________________________________________________________________________________________________
conv2d_13 (Conv2D)              (None, 128, 128, 2)  354         concatenate_6[0][0]              
==================================================================================================
Total params: 11,124,574
Trainable params: 11,122,630
Non-trainable params: 1,944
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x000002308998C5B0>
WARNING:tensorflow:From <ipython-input-7-639783e879e1>:175: Model.fit_generator (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.
Instructions for updating:
Please use Model.fit, which supports generators.
  Training time: 9:40:37.059990
(1515, 128, 128, 1)

Total round time:  9:40:50.260055


*** Leave-one-out round # 1
    Training on 6290 images, validating on 1028 images...
Model: "functional_3"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_2 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d_14 (Conv2D)              (None, 64, 64, 8)    80          input_2[0][0]                    
__________________________________________________________________________________________________
max_pooling2d_7 (MaxPooling2D)  (None, 64, 64, 8)    0           conv2d_14[0][0]                  
__________________________________________________________________________________________________
conv2d_15 (Conv2D)              (None, 32, 32, 16)   1168        max_pooling2d_7[0][0]            
__________________________________________________________________________________________________
max_pooling2d_8 (MaxPooling2D)  (None, 32, 32, 16)   0           conv2d_15[0][0]                  
__________________________________________________________________________________________________
conv2d_16 (Conv2D)              (None, 16, 16, 32)   4640        max_pooling2d_8[0][0]            
__________________________________________________________________________________________________
max_pooling2d_9 (MaxPooling2D)  (None, 16, 16, 32)   0           conv2d_16[0][0]                  
__________________________________________________________________________________________________
conv2d_17 (Conv2D)              (None, 8, 8, 64)     18496       max_pooling2d_9[0][0]            
__________________________________________________________________________________________________
max_pooling2d_10 (MaxPooling2D) (None, 8, 8, 64)     0           conv2d_17[0][0]                  
__________________________________________________________________________________________________
conv2d_18 (Conv2D)              (None, 4, 4, 128)    73856       max_pooling2d_10[0][0]           
__________________________________________________________________________________________________
max_pooling2d_11 (MaxPooling2D) (None, 4, 4, 128)    0           conv2d_18[0][0]                  
__________________________________________________________________________________________________
conv2d_19 (Conv2D)              (None, 2, 2, 256)    295168      max_pooling2d_11[0][0]           
__________________________________________________________________________________________________
max_pooling2d_12 (MaxPooling2D) (None, 2, 2, 256)    0           conv2d_19[0][0]                  
__________________________________________________________________________________________________
conv2d_20 (Conv2D)              (None, 1, 1, 512)    1180160     max_pooling2d_12[0][0]           
__________________________________________________________________________________________________
max_pooling2d_13 (MaxPooling2D) (None, 1, 1, 512)    0           conv2d_20[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_7 (UpSampling2D)  (None, 2, 2, 512)    0           max_pooling2d_13[0][0]           
__________________________________________________________________________________________________
concatenate_7 (Concatenate)     (None, 2, 2, 768)    0           up_sampling2d_7[0][0]            
                                                                 max_pooling2d_12[0][0]           
__________________________________________________________________________________________________
conv2d_21 (Conv2D)              (None, 2, 2, 506)    6218234     concatenate_7[0][0]              
__________________________________________________________________________________________________
batch_normalization_6 (BatchNor (None, 2, 2, 506)    2024        conv2d_21[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_8 (UpSampling2D)  (None, 4, 4, 506)    0           batch_normalization_6[0][0]      
__________________________________________________________________________________________________
concatenate_8 (Concatenate)     (None, 4, 4, 634)    0           up_sampling2d_8[0][0]            
                                                                 max_pooling2d_11[0][0]           
__________________________________________________________________________________________________
conv2d_22 (Conv2D)              (None, 4, 4, 250)    2536250     concatenate_8[0][0]              
__________________________________________________________________________________________________
batch_normalization_7 (BatchNor (None, 4, 4, 250)    1000        conv2d_22[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_9 (UpSampling2D)  (None, 8, 8, 250)    0           batch_normalization_7[0][0]      
__________________________________________________________________________________________________
concatenate_9 (Concatenate)     (None, 8, 8, 314)    0           up_sampling2d_9[0][0]            
                                                                 max_pooling2d_10[0][0]           
__________________________________________________________________________________________________
conv2d_23 (Conv2D)              (None, 8, 8, 122)    613050      concatenate_9[0][0]              
__________________________________________________________________________________________________
batch_normalization_8 (BatchNor (None, 8, 8, 122)    488         conv2d_23[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_10 (UpSampling2D) (None, 16, 16, 122)  0           batch_normalization_8[0][0]      
__________________________________________________________________________________________________
concatenate_10 (Concatenate)    (None, 16, 16, 154)  0           up_sampling2d_10[0][0]           
                                                                 max_pooling2d_9[0][0]            
__________________________________________________________________________________________________
conv2d_24 (Conv2D)              (None, 16, 16, 58)   142970      concatenate_10[0][0]             
__________________________________________________________________________________________________
batch_normalization_9 (BatchNor (None, 16, 16, 58)   232         conv2d_24[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_11 (UpSampling2D) (None, 32, 32, 58)   0           batch_normalization_9[0][0]      
__________________________________________________________________________________________________
concatenate_11 (Concatenate)    (None, 32, 32, 74)   0           up_sampling2d_11[0][0]           
                                                                 max_pooling2d_8[0][0]            
__________________________________________________________________________________________________
conv2d_25 (Conv2D)              (None, 32, 32, 26)   30810       concatenate_11[0][0]             
__________________________________________________________________________________________________
batch_normalization_10 (BatchNo (None, 32, 32, 26)   104         conv2d_25[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_12 (UpSampling2D) (None, 64, 64, 26)   0           batch_normalization_10[0][0]     
__________________________________________________________________________________________________
concatenate_12 (Concatenate)    (None, 64, 64, 34)   0           up_sampling2d_12[0][0]           
                                                                 max_pooling2d_7[0][0]            
__________________________________________________________________________________________________
conv2d_26 (Conv2D)              (None, 64, 64, 10)   5450        concatenate_12[0][0]             
__________________________________________________________________________________________________
batch_normalization_11 (BatchNo (None, 64, 64, 10)   40          conv2d_26[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_13 (UpSampling2D) (None, 128, 128, 10) 0           batch_normalization_11[0][0]     
__________________________________________________________________________________________________
concatenate_13 (Concatenate)    (None, 128, 128, 11) 0           up_sampling2d_13[0][0]           
                                                                 input_2[0][0]                    
__________________________________________________________________________________________________
conv2d_27 (Conv2D)              (None, 128, 128, 2)  354         concatenate_13[0][0]             
==================================================================================================
Total params: 11,124,574
Trainable params: 11,122,630
Non-trainable params: 1,944
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x000002308C3B9370>
  Training time: 10:25:26.385423
(1028, 128, 128, 1)

Total round time:  10:25:35.453333


*** Leave-one-out round # 2
    Training on 6670 images, validating on 648 images...
Model: "functional_5"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_3 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d_28 (Conv2D)              (None, 64, 64, 8)    80          input_3[0][0]                    
__________________________________________________________________________________________________
max_pooling2d_14 (MaxPooling2D) (None, 64, 64, 8)    0           conv2d_28[0][0]                  
__________________________________________________________________________________________________
conv2d_29 (Conv2D)              (None, 32, 32, 16)   1168        max_pooling2d_14[0][0]           
__________________________________________________________________________________________________
max_pooling2d_15 (MaxPooling2D) (None, 32, 32, 16)   0           conv2d_29[0][0]                  
__________________________________________________________________________________________________
conv2d_30 (Conv2D)              (None, 16, 16, 32)   4640        max_pooling2d_15[0][0]           
__________________________________________________________________________________________________
max_pooling2d_16 (MaxPooling2D) (None, 16, 16, 32)   0           conv2d_30[0][0]                  
__________________________________________________________________________________________________
conv2d_31 (Conv2D)              (None, 8, 8, 64)     18496       max_pooling2d_16[0][0]           
__________________________________________________________________________________________________
max_pooling2d_17 (MaxPooling2D) (None, 8, 8, 64)     0           conv2d_31[0][0]                  
__________________________________________________________________________________________________
conv2d_32 (Conv2D)              (None, 4, 4, 128)    73856       max_pooling2d_17[0][0]           
__________________________________________________________________________________________________
max_pooling2d_18 (MaxPooling2D) (None, 4, 4, 128)    0           conv2d_32[0][0]                  
__________________________________________________________________________________________________
conv2d_33 (Conv2D)              (None, 2, 2, 256)    295168      max_pooling2d_18[0][0]           
__________________________________________________________________________________________________
max_pooling2d_19 (MaxPooling2D) (None, 2, 2, 256)    0           conv2d_33[0][0]                  
__________________________________________________________________________________________________
conv2d_34 (Conv2D)              (None, 1, 1, 512)    1180160     max_pooling2d_19[0][0]           
__________________________________________________________________________________________________
max_pooling2d_20 (MaxPooling2D) (None, 1, 1, 512)    0           conv2d_34[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_14 (UpSampling2D) (None, 2, 2, 512)    0           max_pooling2d_20[0][0]           
__________________________________________________________________________________________________
concatenate_14 (Concatenate)    (None, 2, 2, 768)    0           up_sampling2d_14[0][0]           
                                                                 max_pooling2d_19[0][0]           
__________________________________________________________________________________________________
conv2d_35 (Conv2D)              (None, 2, 2, 506)    6218234     concatenate_14[0][0]             
__________________________________________________________________________________________________
batch_normalization_12 (BatchNo (None, 2, 2, 506)    2024        conv2d_35[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_15 (UpSampling2D) (None, 4, 4, 506)    0           batch_normalization_12[0][0]     
__________________________________________________________________________________________________
concatenate_15 (Concatenate)    (None, 4, 4, 634)    0           up_sampling2d_15[0][0]           
                                                                 max_pooling2d_18[0][0]           
__________________________________________________________________________________________________
conv2d_36 (Conv2D)              (None, 4, 4, 250)    2536250     concatenate_15[0][0]             
__________________________________________________________________________________________________
batch_normalization_13 (BatchNo (None, 4, 4, 250)    1000        conv2d_36[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_16 (UpSampling2D) (None, 8, 8, 250)    0           batch_normalization_13[0][0]     
__________________________________________________________________________________________________
concatenate_16 (Concatenate)    (None, 8, 8, 314)    0           up_sampling2d_16[0][0]           
                                                                 max_pooling2d_17[0][0]           
__________________________________________________________________________________________________
conv2d_37 (Conv2D)              (None, 8, 8, 122)    613050      concatenate_16[0][0]             
__________________________________________________________________________________________________
batch_normalization_14 (BatchNo (None, 8, 8, 122)    488         conv2d_37[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_17 (UpSampling2D) (None, 16, 16, 122)  0           batch_normalization_14[0][0]     
__________________________________________________________________________________________________
concatenate_17 (Concatenate)    (None, 16, 16, 154)  0           up_sampling2d_17[0][0]           
                                                                 max_pooling2d_16[0][0]           
__________________________________________________________________________________________________
conv2d_38 (Conv2D)              (None, 16, 16, 58)   142970      concatenate_17[0][0]             
__________________________________________________________________________________________________
batch_normalization_15 (BatchNo (None, 16, 16, 58)   232         conv2d_38[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_18 (UpSampling2D) (None, 32, 32, 58)   0           batch_normalization_15[0][0]     
__________________________________________________________________________________________________
concatenate_18 (Concatenate)    (None, 32, 32, 74)   0           up_sampling2d_18[0][0]           
                                                                 max_pooling2d_15[0][0]           
__________________________________________________________________________________________________
conv2d_39 (Conv2D)              (None, 32, 32, 26)   30810       concatenate_18[0][0]             
__________________________________________________________________________________________________
batch_normalization_16 (BatchNo (None, 32, 32, 26)   104         conv2d_39[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_19 (UpSampling2D) (None, 64, 64, 26)   0           batch_normalization_16[0][0]     
__________________________________________________________________________________________________
concatenate_19 (Concatenate)    (None, 64, 64, 34)   0           up_sampling2d_19[0][0]           
                                                                 max_pooling2d_14[0][0]           
__________________________________________________________________________________________________
conv2d_40 (Conv2D)              (None, 64, 64, 10)   5450        concatenate_19[0][0]             
__________________________________________________________________________________________________
batch_normalization_17 (BatchNo (None, 64, 64, 10)   40          conv2d_40[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_20 (UpSampling2D) (None, 128, 128, 10) 0           batch_normalization_17[0][0]     
__________________________________________________________________________________________________
concatenate_20 (Concatenate)    (None, 128, 128, 11) 0           up_sampling2d_20[0][0]           
                                                                 input_3[0][0]                    
__________________________________________________________________________________________________
conv2d_41 (Conv2D)              (None, 128, 128, 2)  354         concatenate_20[0][0]             
==================================================================================================
Total params: 11,124,574
Trainable params: 11,122,630
Non-trainable params: 1,944
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x0000023094826460>
  Training time: 10:58:40.336138
(648, 128, 128, 1)

Total round time:  10:58:46.489583


*** Leave-one-out round # 3
    Training on 6580 images, validating on 738 images...
Model: "functional_7"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_4 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d_42 (Conv2D)              (None, 64, 64, 8)    80          input_4[0][0]                    
__________________________________________________________________________________________________
max_pooling2d_21 (MaxPooling2D) (None, 64, 64, 8)    0           conv2d_42[0][0]                  
__________________________________________________________________________________________________
conv2d_43 (Conv2D)              (None, 32, 32, 16)   1168        max_pooling2d_21[0][0]           
__________________________________________________________________________________________________
max_pooling2d_22 (MaxPooling2D) (None, 32, 32, 16)   0           conv2d_43[0][0]                  
__________________________________________________________________________________________________
conv2d_44 (Conv2D)              (None, 16, 16, 32)   4640        max_pooling2d_22[0][0]           
__________________________________________________________________________________________________
max_pooling2d_23 (MaxPooling2D) (None, 16, 16, 32)   0           conv2d_44[0][0]                  
__________________________________________________________________________________________________
conv2d_45 (Conv2D)              (None, 8, 8, 64)     18496       max_pooling2d_23[0][0]           
__________________________________________________________________________________________________
max_pooling2d_24 (MaxPooling2D) (None, 8, 8, 64)     0           conv2d_45[0][0]                  
__________________________________________________________________________________________________
conv2d_46 (Conv2D)              (None, 4, 4, 128)    73856       max_pooling2d_24[0][0]           
__________________________________________________________________________________________________
max_pooling2d_25 (MaxPooling2D) (None, 4, 4, 128)    0           conv2d_46[0][0]                  
__________________________________________________________________________________________________
conv2d_47 (Conv2D)              (None, 2, 2, 256)    295168      max_pooling2d_25[0][0]           
__________________________________________________________________________________________________
max_pooling2d_26 (MaxPooling2D) (None, 2, 2, 256)    0           conv2d_47[0][0]                  
__________________________________________________________________________________________________
conv2d_48 (Conv2D)              (None, 1, 1, 512)    1180160     max_pooling2d_26[0][0]           
__________________________________________________________________________________________________
max_pooling2d_27 (MaxPooling2D) (None, 1, 1, 512)    0           conv2d_48[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_21 (UpSampling2D) (None, 2, 2, 512)    0           max_pooling2d_27[0][0]           
__________________________________________________________________________________________________
concatenate_21 (Concatenate)    (None, 2, 2, 768)    0           up_sampling2d_21[0][0]           
                                                                 max_pooling2d_26[0][0]           
__________________________________________________________________________________________________
conv2d_49 (Conv2D)              (None, 2, 2, 506)    6218234     concatenate_21[0][0]             
__________________________________________________________________________________________________
batch_normalization_18 (BatchNo (None, 2, 2, 506)    2024        conv2d_49[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_22 (UpSampling2D) (None, 4, 4, 506)    0           batch_normalization_18[0][0]     
__________________________________________________________________________________________________
concatenate_22 (Concatenate)    (None, 4, 4, 634)    0           up_sampling2d_22[0][0]           
                                                                 max_pooling2d_25[0][0]           
__________________________________________________________________________________________________
conv2d_50 (Conv2D)              (None, 4, 4, 250)    2536250     concatenate_22[0][0]             
__________________________________________________________________________________________________
batch_normalization_19 (BatchNo (None, 4, 4, 250)    1000        conv2d_50[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_23 (UpSampling2D) (None, 8, 8, 250)    0           batch_normalization_19[0][0]     
__________________________________________________________________________________________________
concatenate_23 (Concatenate)    (None, 8, 8, 314)    0           up_sampling2d_23[0][0]           
                                                                 max_pooling2d_24[0][0]           
__________________________________________________________________________________________________
conv2d_51 (Conv2D)              (None, 8, 8, 122)    613050      concatenate_23[0][0]             
__________________________________________________________________________________________________
batch_normalization_20 (BatchNo (None, 8, 8, 122)    488         conv2d_51[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_24 (UpSampling2D) (None, 16, 16, 122)  0           batch_normalization_20[0][0]     
__________________________________________________________________________________________________
concatenate_24 (Concatenate)    (None, 16, 16, 154)  0           up_sampling2d_24[0][0]           
                                                                 max_pooling2d_23[0][0]           
__________________________________________________________________________________________________
conv2d_52 (Conv2D)              (None, 16, 16, 58)   142970      concatenate_24[0][0]             
__________________________________________________________________________________________________
batch_normalization_21 (BatchNo (None, 16, 16, 58)   232         conv2d_52[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_25 (UpSampling2D) (None, 32, 32, 58)   0           batch_normalization_21[0][0]     
__________________________________________________________________________________________________
concatenate_25 (Concatenate)    (None, 32, 32, 74)   0           up_sampling2d_25[0][0]           
                                                                 max_pooling2d_22[0][0]           
__________________________________________________________________________________________________
conv2d_53 (Conv2D)              (None, 32, 32, 26)   30810       concatenate_25[0][0]             
__________________________________________________________________________________________________
batch_normalization_22 (BatchNo (None, 32, 32, 26)   104         conv2d_53[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_26 (UpSampling2D) (None, 64, 64, 26)   0           batch_normalization_22[0][0]     
__________________________________________________________________________________________________
concatenate_26 (Concatenate)    (None, 64, 64, 34)   0           up_sampling2d_26[0][0]           
                                                                 max_pooling2d_21[0][0]           
__________________________________________________________________________________________________
conv2d_54 (Conv2D)              (None, 64, 64, 10)   5450        concatenate_26[0][0]             
__________________________________________________________________________________________________
batch_normalization_23 (BatchNo (None, 64, 64, 10)   40          conv2d_54[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_27 (UpSampling2D) (None, 128, 128, 10) 0           batch_normalization_23[0][0]     
__________________________________________________________________________________________________
concatenate_27 (Concatenate)    (None, 128, 128, 11) 0           up_sampling2d_27[0][0]           
                                                                 input_4[0][0]                    
__________________________________________________________________________________________________
conv2d_55 (Conv2D)              (None, 128, 128, 2)  354         concatenate_27[0][0]             
==================================================================================================
Total params: 11,124,574
Trainable params: 11,122,630
Non-trainable params: 1,944
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x000002308C18B790>
  Training time: 10:51:03.721528
(738, 128, 128, 1)

Total round time:  10:51:10.340107


*** Leave-one-out round # 4
    Training on 6689 images, validating on 629 images...
Model: "functional_9"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_5 (InputLayer)            [(None, 128, 128, 1) 0                                            
__________________________________________________________________________________________________
conv2d_56 (Conv2D)              (None, 64, 64, 8)    80          input_5[0][0]                    
__________________________________________________________________________________________________
max_pooling2d_28 (MaxPooling2D) (None, 64, 64, 8)    0           conv2d_56[0][0]                  
__________________________________________________________________________________________________
conv2d_57 (Conv2D)              (None, 32, 32, 16)   1168        max_pooling2d_28[0][0]           
__________________________________________________________________________________________________
max_pooling2d_29 (MaxPooling2D) (None, 32, 32, 16)   0           conv2d_57[0][0]                  
__________________________________________________________________________________________________
conv2d_58 (Conv2D)              (None, 16, 16, 32)   4640        max_pooling2d_29[0][0]           
__________________________________________________________________________________________________
max_pooling2d_30 (MaxPooling2D) (None, 16, 16, 32)   0           conv2d_58[0][0]                  
__________________________________________________________________________________________________
conv2d_59 (Conv2D)              (None, 8, 8, 64)     18496       max_pooling2d_30[0][0]           
__________________________________________________________________________________________________
max_pooling2d_31 (MaxPooling2D) (None, 8, 8, 64)     0           conv2d_59[0][0]                  
__________________________________________________________________________________________________
conv2d_60 (Conv2D)              (None, 4, 4, 128)    73856       max_pooling2d_31[0][0]           
__________________________________________________________________________________________________
max_pooling2d_32 (MaxPooling2D) (None, 4, 4, 128)    0           conv2d_60[0][0]                  
__________________________________________________________________________________________________
conv2d_61 (Conv2D)              (None, 2, 2, 256)    295168      max_pooling2d_32[0][0]           
__________________________________________________________________________________________________
max_pooling2d_33 (MaxPooling2D) (None, 2, 2, 256)    0           conv2d_61[0][0]                  
__________________________________________________________________________________________________
conv2d_62 (Conv2D)              (None, 1, 1, 512)    1180160     max_pooling2d_33[0][0]           
__________________________________________________________________________________________________
max_pooling2d_34 (MaxPooling2D) (None, 1, 1, 512)    0           conv2d_62[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_28 (UpSampling2D) (None, 2, 2, 512)    0           max_pooling2d_34[0][0]           
__________________________________________________________________________________________________
concatenate_28 (Concatenate)    (None, 2, 2, 768)    0           up_sampling2d_28[0][0]           
                                                                 max_pooling2d_33[0][0]           
__________________________________________________________________________________________________
conv2d_63 (Conv2D)              (None, 2, 2, 506)    6218234     concatenate_28[0][0]             
__________________________________________________________________________________________________
batch_normalization_24 (BatchNo (None, 2, 2, 506)    2024        conv2d_63[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_29 (UpSampling2D) (None, 4, 4, 506)    0           batch_normalization_24[0][0]     
__________________________________________________________________________________________________
concatenate_29 (Concatenate)    (None, 4, 4, 634)    0           up_sampling2d_29[0][0]           
                                                                 max_pooling2d_32[0][0]           
__________________________________________________________________________________________________
conv2d_64 (Conv2D)              (None, 4, 4, 250)    2536250     concatenate_29[0][0]             
__________________________________________________________________________________________________
batch_normalization_25 (BatchNo (None, 4, 4, 250)    1000        conv2d_64[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_30 (UpSampling2D) (None, 8, 8, 250)    0           batch_normalization_25[0][0]     
__________________________________________________________________________________________________
concatenate_30 (Concatenate)    (None, 8, 8, 314)    0           up_sampling2d_30[0][0]           
                                                                 max_pooling2d_31[0][0]           
__________________________________________________________________________________________________
conv2d_65 (Conv2D)              (None, 8, 8, 122)    613050      concatenate_30[0][0]             
__________________________________________________________________________________________________
batch_normalization_26 (BatchNo (None, 8, 8, 122)    488         conv2d_65[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_31 (UpSampling2D) (None, 16, 16, 122)  0           batch_normalization_26[0][0]     
__________________________________________________________________________________________________
concatenate_31 (Concatenate)    (None, 16, 16, 154)  0           up_sampling2d_31[0][0]           
                                                                 max_pooling2d_30[0][0]           
__________________________________________________________________________________________________
conv2d_66 (Conv2D)              (None, 16, 16, 58)   142970      concatenate_31[0][0]             
__________________________________________________________________________________________________
batch_normalization_27 (BatchNo (None, 16, 16, 58)   232         conv2d_66[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_32 (UpSampling2D) (None, 32, 32, 58)   0           batch_normalization_27[0][0]     
__________________________________________________________________________________________________
concatenate_32 (Concatenate)    (None, 32, 32, 74)   0           up_sampling2d_32[0][0]           
                                                                 max_pooling2d_29[0][0]           
__________________________________________________________________________________________________
conv2d_67 (Conv2D)              (None, 32, 32, 26)   30810       concatenate_32[0][0]             
__________________________________________________________________________________________________
batch_normalization_28 (BatchNo (None, 32, 32, 26)   104         conv2d_67[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_33 (UpSampling2D) (None, 64, 64, 26)   0           batch_normalization_28[0][0]     
__________________________________________________________________________________________________
concatenate_33 (Concatenate)    (None, 64, 64, 34)   0           up_sampling2d_33[0][0]           
                                                                 max_pooling2d_28[0][0]           
__________________________________________________________________________________________________
conv2d_68 (Conv2D)              (None, 64, 64, 10)   5450        concatenate_33[0][0]             
__________________________________________________________________________________________________
batch_normalization_29 (BatchNo (None, 64, 64, 10)   40          conv2d_68[0][0]                  
__________________________________________________________________________________________________
up_sampling2d_34 (UpSampling2D) (None, 128, 128, 10) 0           batch_normalization_29[0][0]     
__________________________________________________________________________________________________
concatenate_34 (Concatenate)    (None, 128, 128, 11) 0           up_sampling2d_34[0][0]           
                                                                 input_5[0][0]                    
__________________________________________________________________________________________________
conv2d_69 (Conv2D)              (None, 128, 128, 2)  354         concatenate_34[0][0]             
==================================================================================================
Total params: 11,124,574
Trainable params: 11,122,630
Non-trainable params: 1,944
__________________________________________________________________________________________________
TRAINING LOG <ultrasound_batch_generator.UltrasoundSegmentationBatchGenerator object at 0x0000023089E5A940>
  Training time: 11:08:47.302576
(629, 128, 128, 1)

Total round time:  11:08:53.025421


Total training time:   2 days, 5:05:50.763324

# Arrange results in tables

metric_labels = [
    "AUROC",
    "best thresh",
    "best TP",
    "best FP",
    "best recall",
    "best precis",
    "fuzzy recall",
    "fuzzy precis",
    "fuzzy Fscore"
]

results_labels = []
  
for label in metric_labels:
    results_labels.append("Vali " + label)

results_df = pd.DataFrame(columns = results_labels)

for i in range(num_validation_rounds):
    if i in val_best_metrics.keys():
        results_df.loc[i] = [
            val_aurocs[i],
            val_best_thresholds[i],
            val_best_metrics[i][evaluation_metrics.TRUE_POSITIVE_RATE],
            val_best_metrics[i][evaluation_metrics.FALSE_POSITIVE_RATE],
            val_best_metrics[i][evaluation_metrics.RECALL],
            val_best_metrics[i][evaluation_metrics.PRECISION],
            val_fuzzy_metrics[i][evaluation_metrics.RECALL],
            val_fuzzy_metrics[i][evaluation_metrics.PRECISION],
            val_fuzzy_metrics[i][evaluation_metrics.FSCORE]
        ]

display(results_df)

print("\nAverages")

results_means_df = results_df.mean()
display(results_means_df)

	Vali AUROC	Vali best thresh	Vali best TP	Vali best FP	Vali best recall	Vali best precis	Vali fuzzy recall	Vali fuzzy precis	Vali fuzzy Fscore
0	0.964690	0.001	0.973915	0.057440	0.973915	0.125060	0.758485	0.392868	0.517625
1	0.582555	0.001	0.620815	0.175993	0.620815	0.042935	0.277600	0.169212	0.210259
2	0.952046	0.010	0.954157	0.096753	0.954157	0.179243	0.724504	0.325453	0.449146
3	0.842104	0.001	0.863717	0.098457	0.863717	0.208607	0.466506	0.528203	0.495441
4	0.766165	0.001	0.793075	0.122831	0.793075	0.186265	0.411286	0.489370	0.446943

Averages

Vali AUROC           0.821512
Vali best thresh     0.002800
Vali best TP         0.841136
Vali best FP         0.110295
Vali best recall     0.841136
Vali best precis     0.148422
Vali fuzzy recall    0.527676
Vali fuzzy precis    0.381021
Vali fuzzy Fscore    0.423883
dtype: float64

# Print the last ROC curve for visual verification that we catch the optimal point

n = len(roc_thresholds)

roc_x = np.zeros(n)
roc_y = np.zeros(n)

for i in range(n):
    roc_x[i] = vali_metrics_dicts[i][evaluation_metrics.FALSE_POSITIVE_RATE]
    roc_y[i] = vali_metrics_dicts[i][evaluation_metrics.SENSITIVITY]
    # print("Threshold = {0:4.2f}  False pos rate = {1:4.2f}  Sensitivity = {2:4.2f}"
    #       .format(roc_thresholds[i], roc_x[i], roc_y[i]))

    
plt.figure()
plt.ylim(-0.01, 1.01)
plt.xlim(-0.01, 1.01)
plt.plot(roc_x, roc_y, color='darkred', lw=2)

[<matplotlib.lines.Line2D at 0x2308d8be8e0>]

# Save results table

csv_filename = this_notebook_name + "_" + save_timestamp + ".csv"
csv_fullname = os.path.join(results_save_fullpath, csv_filename)
results_df.to_csv(csv_fullname)

print("Results saved to: {}".format(csv_fullname))

Results saved to: /Usersß/Josh Ehrlich/Courses/CISC881/Project/data\SavedResults\BreastSegmentationStudy-TF2_2020-11-27_22-59-01.csv

# Display sample results

# determine the available ultrasounds in the last round of validation
last_round = validation_schedule_patient[-1]
print(val_ultrasound_data.shape)
prev_vali = 0

#20: [63, 94, 72, 121, 130, 2, 64, 111, 26, 46, 12, 67, 132, 49, 61, 90, 65, 106, 14, 100]
#23: [347, 185, 215, 228, 345, 176, 186, 335, 301, 237, 316, 288, 181, 354, 331, 261, 194, 225, 205, 153]
#27: [364, 459, 405, 464, 426, 397, 483, 416, 430, 392, 447, 401, 423, 409, 378, 458, 422, 445, 451, 379]
#30: [494, 488, 571, 626, 568, 529, 582, 486, 484, 533, 485, 574, 507, 516, 493, 499, 549, 561, 625, 523]
count = 0
patientNumber = [20, 23, 27, 30]
sample_indices = [[63, 94, 72, 121, 130, 2, 64, 111, 26, 46, 12, 67, 132, 49, 61, 90, 65, 106, 14, 100], [347, 185, 215, 228, 345, 176, 186, 335, 301, 237, 316, 288, 181, 354, 331, 261, 194, 225, 205, 153], [364, 459, 405, 464, 426, 397, 483, 416, 430, 392, 447, 401, 423, 409, 378, 458, 422, 445, 451, 379], [494, 488, 571, 626, 568, 529, 582, 486, 484, 533, 485, 574, 507, 516, 493, 499, 549, 561, 625, 523]]
for j in range(len(last_round)):
    
    num_vali =  prev_vali + ultrasound_arrays_by_patients[last_round[j]].shape[0]
    num_show = 20
    if num_vali < num_show:
        num_show = 0
    num_col = 4

    indices = [i for i in range(prev_vali, num_vali)]
    #sample_indices = sample(indices, num_show)
    print("Showing image slices for patient", patientNumber[count], ":", sample_indices[count])
    #sample_indices = [prev_vali]
    
    # update
    prev_vali = num_vali
    threshold = 0.5
    
    fig = plt.figure(figsize=(18, num_show*5))
    for i in range(num_show):
        a0 = fig.add_subplot(num_show, num_col, i*num_col+1)
        img0 = a0.imshow(np.flipud(val_ultrasound_data[sample_indices[count][i], :, :, 0].astype(np.float32)))
        a0.set_title("Patient #{} - Ultrasound #{}".format(last_round[j], sample_indices[count][i]))
        a1 = fig.add_subplot(num_show, num_col, i*num_col+2)
        img1 = a1.imshow(np.flipud(val_segmentation_data_onehot[sample_indices[count][i], :, :, 1]), vmin=0.0, vmax=1.0)
        a1.set_title("Segmentation #{}".format(sample_indices[count][i]))
        c = fig.colorbar(img1, fraction=0.046, pad=0.04)
        a2 = fig.add_subplot(num_show, num_col, i*num_col+3)
        img2 = a2.imshow(np.flipud(y_pred_val[sample_indices[count][i], :, :, 1]), vmin=0.0, vmax=1.0)
        a2.set_title("Prediction #{}".format(sample_indices[count][i]))
        c = fig.colorbar(img2, fraction=0.046, pad=0.04)
        a3 = fig.add_subplot(num_show, num_col, i*num_col+4)
        img3 = a3.imshow((np.flipud(y_pred_val[sample_indices[count][i], :, :, 1]) > threshold), vmin=0.0, vmax=1.0)
        c = fig.colorbar(img3, fraction=0.046, pad=0.04)
        a3.set_title("Thresholded #{}".format(sample_indices[count][i]))
    count = count + 1

(629, 128, 128, 1)
Showing image slices for patient 20 : [63, 94, 72, 121, 130, 2, 64, 111, 26, 46, 12, 67, 132, 49, 61, 90, 65, 106, 14, 100]
Showing image slices for patient 23 : [347, 185, 215, 228, 345, 176, 186, 335, 301, 237, 316, 288, 181, 354, 331, 261, 194, 225, 205, 153]
Showing image slices for patient 27 : [364, 459, 405, 464, 426, 397, 483, 416, 430, 392, 447, 401, 423, 409, 378, 458, 422, 445, 451, 379]
Showing image slices for patient 30 : [494, 488, 571, 626, 568, 529, 582, 486, 484, 533, 485, 574, 507, 516, 493, 499, 549, 561, 625, 523]

# Save notebook so all output is archived by the next cell

from IPython.display import Javascript
script = '''
require(["base/js/namespace"],function(Jupyter) {
    Jupyter.notebook.save_checkpoint();
});
'''
Javascript(script)

# Export HTML copy of this notebook

notebook_file_name =  "/Users/Josh Ehrlich/Courses/CISC881/Project/data/"+this_notebook_name + "_" + save_timestamp + ".html"
notebook_fullname = os.path.join(notebooks_save_fullpath, notebook_file_name)

os.system("jupyter nbconvert --to html " + this_notebook_name + " --output " + notebook_fullname)
print("Notebook saved to: {}".format(notebook_fullname))

notebook_file_name

print("Notebook_name", this_notebook_name)
print("fullname", notebook_fullname)